package com.stack.wemedia.crawler;

import com.alibaba.fastjson.JSON;
import com.stack.apis.admin.IAdminClient;
import com.stack.common.utils.thread.WmThreadLocalUtil;
import com.stack.file.service.FileStorageService;
import com.stack.model.admin.dtos.AdNewsDto;
import com.stack.model.wemedia.dtos.WmNewsDto;
import com.stack.model.wemedia.pojos.NewsCrawler;
import com.stack.model.wemedia.pojos.WmMaterial;
import com.stack.model.wemedia.pojos.WmUser;
import com.stack.wemedia.WemediaApplication;
import com.stack.wemedia.mapper.WmMaterialMapper;
import com.stack.wemedia.service.NewsCrawlerService;
import com.stack.wemedia.service.WmNewsService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.tomcat.util.http.fileupload.IOUtils;
import org.assertj.core.util.Lists;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import javax.annotation.Resource;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.*;

/**
 * @author llz
 * @date 2023/12/4 11:05
 * @description 博客园爬取新闻数据
 */
@SpringBootTest(classes = WemediaApplication.class)
@RunWith(SpringRunner.class)
@Slf4j
public class BlogCrawler {
    @Resource
    private NewsCrawlerService newsCrawlerService;
    @Resource
    private WmNewsService wmNewsService;
    @Resource
    private WmMaterialMapper wmMaterialMapper;
    @Resource
    private FileStorageService fileStorageService;
    @Resource
    private IAdminClient adminClient;

    @Test
    public void test() {
//        List<String> materials = new ArrayList<>();
//        materials.add("http://192.168.200.130:9000/leadnews/2023/12/01/316558ee5cbf4b8db84c1f0e601b8682.gif");
//        List<WmMaterial> dbMaterials = wmMaterialMapper.selectList(Wrappers.<WmMaterial>lambdaQuery().in(WmMaterial::getUrl, materials));
//        log.info("通过图片的url查询素材的id：{}", dbMaterials);

        List<String> urls = newsCrawlerService.getUrlByChannel(1);
        log.info("获取的url：{}", urls);
        for (String url : urls) {
            NewsCrawler newsCrawler = new NewsCrawler();
            newsCrawler.setUrl(url);
            newsCrawler.setStatus(1);
            newsCrawlerService.update(newsCrawler);
        }

//        NewsCrawler newsCrawler = new NewsCrawler();
//        newsCrawler.setTitle("title");
//        newsCrawler.setUrl("newsUrl");
//        newsCrawler.setChannel(1);
//        newsCrawler.setStatus(0);
//        newsCrawler.setCreatedTime(new Date());
//        log.info("插入数据:{}", JSON.toJSONString(newsCrawler));
//        Integer i = newsCrawlerService.insert(newsCrawler);
//        log.info("结果:{}", i);
    }

    @Test
    public void testGetDetails() throws IOException {
        String url = "https://www.cnblogs.com/Jcloud/p/17874259.html";
        Document document = Jsoup.connect(url).get();
        log.info("document: {}", document);
        Element content = document.getElementsByClass("postBody").get(0);

        Elements elements = content.getAllElements();
        for (Element element : elements) {
            if (element.tagName().equals("p")) {
                log.info("新闻内容: {}", element.text());
            }
            if (element.tagName().equals("img")) {
                String imgUrl = element.attr("src");
                log.info("新闻图片: {}", imgUrl);
            }
        }


//        Element element = document.getElementsByClass("blogpost-body blogpost-body-html").get(0);
//        Elements contents = element.getElementsByTag("span");
//        Elements imgs = element.getElementsByTag("img");

//        Elements elements = content.getElementsByTag("P");
//        for (Element element : elements) {
//            String text = element.text();
//            String img = element.getElementsByTag("img").attr("src");
//            log.info("新闻内容: {}", text);
//            log.info("新闻图片: {}", img);
//        }
    }

    /**
     * 获取频道每条新闻的url
     */
    @Test
    public void getNewsUrl() throws IOException {
        // CSDN
//        String url = "https://www.csdn.net/?spm=1035.2022.3001.4476";
//        String url = "https://blog.csdn.net/rank/list";
        //博客园
//        https://www.cnblogs.com/xfuture/
//        String url = "https://www.cnblogs.com/cate/2/#p2";
//        String url = "https://www.cnblogs.com/cate/java/";
        String url = "https://www.cnblogs.com/cate/java/#p1";

        // 获取该网页document文档数据
        Document document = Jsoup.connect(url)
                .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36")
                .get();

        log.info("document: {}", document);


        Elements elements = document.getElementsByClass("post-item-title");
        log.info("文章url数量: {}", elements.size());
        for (Element element : elements) {
            String newsUrl = element.attr("href");
            String title = element.text();
//            if (title.length() > 30) {
//                title = title.substring(0, 29);
//            }
            log.info("新闻的url: {}", newsUrl);
            log.info("标题: {}", title);

            try {
                NewsCrawler newsCrawler = new NewsCrawler();
                newsCrawler.setTitle(title);
                newsCrawler.setUrl(newsUrl);
                newsCrawler.setChannel(2);
                newsCrawler.setStatus(0);
                newsCrawler.setCreatedTime(new Date());
                newsCrawlerService.insert(newsCrawler);
                log.info("爬爬爬爬爬爬爬!");
            } catch (Exception e) {
                log.error("标题不合规，爬取失败: {}", e.getMessage());
            }
        }
    }

    /**
     * 获取新闻细节
     */
    @Test
    public void getNewsDetail() throws IOException {
        // 模拟当前自媒体登录用户
        WmUser wmUser = new WmUser();
        wmUser.setId(1103);
        WmThreadLocalUtil.setUser(wmUser);

        List<NewsCrawler> newsList = newsCrawlerService.getNews(2);
        log.info("本次爬取的url数量: {}", newsList.size());

        for (NewsCrawler news : newsList) {
            String url = news.getUrl();
            String title = news.getTitle();
            log.info("本次爬取url: {}", url);

            // 获取该网页document文档数据
            Document document = null;
            Element content = null;
            try {
                document = Jsoup.connect(url).get();
                content = document.getElementsByClass("postBody").get(0);
                log.info("document: {}", document);
            } catch (Exception e) {
                log.error("获取新闻详情失败,原因是{}", e.getMessage());
                continue;
            }


            Elements elements = content.getAllElements();
            List<Map> contentMap = new ArrayList<>();
            List<String> imgUrlList = new ArrayList<>();
            for (Element element : elements) {
                Map<String, String> map = null;
//                if (StringUtils.isNotEmpty(element.attr("src"))) {
//                    continue;
//                }
                try {
                    if (element.tagName().equals("p")) {
                        String text = element.text();
                        log.info("新闻内容: {}", text);
                        map = new HashMap<>();
                        map.put("type", "text");
                        map.put("value", text);
                        contentMap.add(map);
                    }
                    if (element.tagName().equals("img")) {
                        String imgUrl = element.attr("src");
                        log.info("新闻图片: {}", imgUrl);
                        String fileName = uploadPic(imgUrl);
                        imgUrlList.add(fileName);
                        map = new HashMap<>();
                        map.put("type", "image");
                        map.put("value", fileName);
                        contentMap.add(map);
                    }
                } catch (Exception e) {
                    log.error("解析失败,原因是{}", e.getMessage());
                    continue;
                }
            }

            log.info("解析后的图片: {}", imgUrlList);
            log.info("解析后的内容: {}", contentMap);

            // 上传图片//保存素材
            for (String imgUrl : imgUrlList) {
                WmMaterial wmMaterial = null;
                if (StringUtils.isNotBlank(imgUrl)) {
                    // 如果上传图片路径不为空  创建素材信息
                    wmMaterial = new WmMaterial();
                    wmMaterial.setUserId(WmThreadLocalUtil.getUser().getId());
                    wmMaterial.setUrl(imgUrl);
                    wmMaterial.setType((short) 0);
                    wmMaterial.setIsCollection((short) 0);
                    wmMaterial.setCreatedTime(new Date());
                    log.info("插入素材信息: {}", wmMaterial);
                    wmMaterialMapper.insert(wmMaterial);
                }
            }

            try {
                // 保存自媒体文章
                WmNewsDto wmNewsDto = new WmNewsDto();
                wmNewsDto.setContent(JSON.toJSONString(contentMap));
                wmNewsDto.setTitle(title);
                log.info("文章封面: {}", imgUrlList);
                if (imgUrlList.size() > 0) {
                    wmNewsDto.setType((short) 1);
                    List<String> imgUrls = Lists.newArrayList(imgUrlList.get(0));
                    wmNewsDto.setImages(imgUrls);
                } else {
                    wmNewsDto.setType((short) 0);
                }

                wmNewsDto.setChannelId(1);
                try {
                    Thread.sleep(1000); // 睡眠1秒 让发布时间不一致
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                wmNewsDto.setPublishTime(new Date());
                wmNewsDto.setStatus((short) 9); //
                wmNewsDto.setLabels("采集");
                log.info("保存自媒体文章: {}", wmNewsDto);
                wmNewsService.submitNews(wmNewsDto);

                // 保存admin端的相关的审核文章数据
                AdNewsDto dto = new AdNewsDto();
                dto.setUserId(WmThreadLocalUtil.getUser().getId());
                dto.setAuthorName("crawler菜鸟");
                dto.setTitle(title);
                dto.setContent(JSON.toJSONString(contentMap));
                dto.setStatus(3);
                dto.setChannelId(1);
                dto.setLabels("采集");
                dto.setCreatedTime(new Date());
                dto.setSubmitedTime(new Date());
                if (imgUrlList.size() > 0) {
                    dto.setType(1);
                    List<String> imgUrls = Lists.newArrayList(imgUrlList.get(0));
                    dto.setImages(imgUrls);
                } else {
                    dto.setType(0);
                }
                dto.setEnable(0);
                log.info("保存admin端的相关的审核文章数据: {}", dto);
                adminClient.saveAdCheckNews(dto);
            } catch (Exception e) {
                log.error("保存自媒体文章失败:{}", e.getMessage());
                log.error("保存admin端的相关的审核文章数据失败");
            } finally {
                //修改爬虫配置表
                NewsCrawler newsCrawler = new NewsCrawler();
                newsCrawler.setUrl(url);
                newsCrawler.setStatus(1);
                newsCrawlerService.update(newsCrawler);
            }
        }

    }

    /**
     * 上传图片
     * @param imgUrl
     * @return
     */
    public String uploadPic(String imgUrl) {
        String uuid = UUID.randomUUID().toString().replace("-", "");
        String suffix = imgUrl.substring(imgUrl.lastIndexOf("."));
        InputStream in = getInputStreamByUrl(imgUrl);
        if (in != null) {
            String fileName = fileStorageService.uploadImgFile("", uuid + suffix, in);
            System.out.println("上传文件名称: " + fileName);
            return fileName;
        }
        return null;
    }

    /**
     * 工具方法:  根据url路径 获取输入流
     *
     * @param strUrl
     * @return
     */
    public static InputStream getInputStreamByUrl(String strUrl) {
        HttpURLConnection conn = null;
        try {
            URL url = new URL(strUrl);
            conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("GET");
            conn.setConnectTimeout(20 * 1000);
            final ByteArrayOutputStream output = new ByteArrayOutputStream();
            IOUtils.copy(conn.getInputStream(), output);
            return new ByteArrayInputStream(output.toByteArray());
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                if (conn != null) {
                    conn.disconnect();
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        return null;
    }


}